{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learn More About How to Build & Train Neural Networks\n",
    "\n",
    " - Learning from Karpathy's Micrograd, try to code it myself.\n",
    " - We use scalars instead of tensors to keep things simple. However, tensor operations are just convenient ways to perform batches of scalar operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scalar(val=3, grad=0)\n",
      "Scalar(val=6, grad=0)\n"
     ]
    }
   ],
   "source": [
    "from tinyflow import Scalar\n",
    "\n",
    "a = Scalar(1)\n",
    "b = Scalar(2)\n",
    "c = a + b\n",
    "print(c)\n",
    "\n",
    "c = c * b\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"794pt\" height=\"128pt\"\n",
       " viewBox=\"0.00 0.00 794.00 128.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 124)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-124 790,-124 790,4 -4,4\"/>\n",
       "<!-- 280619742520832 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>280619742520832</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-83.5 0,-119.5 178,-119.5 178,-83.5 0,-83.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 2.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-83.5 83,-119.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-97.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.000</text>\n",
       "</g>\n",
       "<!-- 280619378504672* -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>280619378504672*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"241\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"241\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 280619742520832&#45;&gt;280619378504672* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>280619742520832&#45;&gt;280619378504672*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.12,-85.05C187.54,-83.29 196.67,-81.59 204.9,-80.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"205.6,-83.48 214.79,-78.21 204.32,-76.6 205.6,-83.48\"/>\n",
       "</g>\n",
       "<!-- 280619378505776 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>280619378505776</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"608,-27.5 608,-63.5 786,-63.5 786,-27.5 608,-27.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"649.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 8.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"691,-27.5 691,-63.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"738.5\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.000</text>\n",
       "</g>\n",
       "<!-- 280619378505776+ -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>280619378505776+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"545\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"545\" y=\"-41.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 280619378505776+&#45;&gt;280619378505776 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>280619378505776+&#45;&gt;280619378505776</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M572.13,-45.5C579.65,-45.5 588.33,-45.5 597.54,-45.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"597.7,-49 607.7,-45.5 597.7,-42 597.7,-49\"/>\n",
       "</g>\n",
       "<!-- 280619742769760 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>280619742769760</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"304,-0.5 304,-36.5 482,-36.5 482,-0.5 304,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"345.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 6.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"387,-0.5 387,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"434.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.000</text>\n",
       "</g>\n",
       "<!-- 280619742769760&#45;&gt;280619378505776+ -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>280619742769760&#45;&gt;280619378505776+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M482.12,-34.36C491.54,-36.06 500.67,-37.7 508.9,-39.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"508.33,-42.64 518.79,-40.96 509.57,-35.75 508.33,-42.64\"/>\n",
       "</g>\n",
       "<!-- 280619742378656 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>280619742378656</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-28.5 0,-64.5 178,-64.5 178,-28.5 0,-28.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 1.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-28.5 83,-64.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-42.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.000</text>\n",
       "</g>\n",
       "<!-- 280619742378656&#45;&gt;280619378504672* -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>280619742378656&#45;&gt;280619378504672*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.12,-62.36C187.54,-64.06 196.67,-65.7 204.9,-67.18\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"204.33,-70.64 214.79,-68.96 205.57,-63.75 204.33,-70.64\"/>\n",
       "</g>\n",
       "<!-- 280619378504672 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>280619378504672</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"304,-55.5 304,-91.5 482,-91.5 482,-55.5 304,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"345.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 2.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"387,-55.5 387,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"434.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 0.000</text>\n",
       "</g>\n",
       "<!-- 280619378504672&#45;&gt;280619378505776+ -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>280619378504672&#45;&gt;280619378505776+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M482.12,-57.05C491.54,-55.29 500.67,-53.59 508.9,-52.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"509.6,-55.48 518.79,-50.21 508.32,-48.6 509.6,-55.48\"/>\n",
       "</g>\n",
       "<!-- 280619378504672*&#45;&gt;280619378504672 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>280619378504672*&#45;&gt;280619378504672</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M268.13,-73.5C275.65,-73.5 284.33,-73.5 293.54,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"293.7,-77 303.7,-73.5 293.7,-70 293.7,-77\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0xff38ca5f7920>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# draw the flow, copy from Karpathy's Micrograd\n",
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for child in v._prev:\n",
    "                edges.add((child, v))\n",
    "                build(child)\n",
    "    build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "def draw_dot(root, format='svg', rankdir='LR'):\n",
    "    \"\"\"\n",
    "    format: png | svg | ...\n",
    "    rankdir: TB (top to bottom graph) | LR (left to right)\n",
    "    \"\"\"\n",
    "    assert rankdir in ['LR', 'TB']\n",
    "    nodes, edges = trace(root)\n",
    "    dot = Digraph(format=format, graph_attr={'rankdir': rankdir}) #, node_attr={'rankdir': 'TB'})\n",
    "    \n",
    "    for n in nodes:\n",
    "        dot.node(name=str(id(n)), label = \"{ val %.3f | grad %.3f }\" % (n.val, n.grad), shape='record')\n",
    "        if n._op:\n",
    "            dot.node(name=str(id(n)) + n._op, label=n._op)\n",
    "            dot.edge(str(id(n)) + n._op, str(id(n)))\n",
    "    \n",
    "    for n1, n2 in edges:\n",
    "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "    \n",
    "    return dot\n",
    "\n",
    "c = Scalar(6)\n",
    "\n",
    "d = a * b + c\n",
    "draw_dot(d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"1457pt\" height=\"210pt\"\n",
       " viewBox=\"0.00 0.00 1456.79 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-206 1452.79,-206 1452.79,4 -4,4\"/>\n",
       "<!-- 280619742519824 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>280619742519824</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"304,-55.5 304,-91.5 491,-91.5 491,-55.5 304,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"350\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 15.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"396,-55.5 396,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"443.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742520544+ -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>280619742520544+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"554\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"554\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 280619742519824&#45;&gt;280619742520544+ -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>280619742519824&#45;&gt;280619742520544+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M491.05,-89.67C500.34,-91.3 509.33,-92.87 517.44,-94.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"517.1,-97.78 527.55,-96.05 518.3,-90.88 517.1,-97.78\"/>\n",
       "</g>\n",
       "<!-- 280619742519824* -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>280619742519824*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"241\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"241\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 280619742519824*&#45;&gt;280619742519824 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>280619742519824*&#45;&gt;280619742519824</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M268.21,-73.5C275.73,-73.5 284.43,-73.5 293.67,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"293.89,-77 303.89,-73.5 293.89,-70 293.89,-77\"/>\n",
       "</g>\n",
       "<!-- 280619742376496 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>280619742376496</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-165.5 0,-201.5 178,-201.5 178,-165.5 0,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 2.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-165.5 83,-201.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-179.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 4.000</text>\n",
       "</g>\n",
       "<!-- 280619742525008* -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>280619742525008*</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"241\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"241\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">*</text>\n",
       "</g>\n",
       "<!-- 280619742376496&#45;&gt;280619742525008* -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>280619742376496&#45;&gt;280619742525008*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M152.35,-165.44C161.03,-162.62 169.78,-159.6 178,-156.5 188.47,-152.55 199.68,-147.69 209.65,-143.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"211.17,-146.28 218.76,-138.9 208.22,-139.94 211.17,-146.28\"/>\n",
       "</g>\n",
       "<!-- 280619742525008 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>280619742525008</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"308.5,-110.5 308.5,-146.5 486.5,-146.5 486.5,-110.5 308.5,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"350\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 8.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"391.5,-110.5 391.5,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"439\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742525008&#45;&gt;280619742520544+ -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>280619742525008&#45;&gt;280619742520544+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M486.58,-112.54C497.51,-110.56 508.15,-108.63 517.59,-106.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"518.48,-110.31 527.69,-105.09 517.23,-103.43 518.48,-110.31\"/>\n",
       "</g>\n",
       "<!-- 280619742525008*&#45;&gt;280619742525008 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>280619742525008*&#45;&gt;280619742525008</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M268.21,-128.5C276.91,-128.5 287.17,-128.5 298.05,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"298.17,-132 308.17,-128.5 298.17,-125 298.17,-132\"/>\n",
       "</g>\n",
       "<!-- 280619742517376 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>280619742517376</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"930,-109.5 930,-145.5 1117,-145.5 1117,-109.5 930,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"976\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 29.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1022,-109.5 1022,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1069.5\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742520112ReLU -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>280619742520112ReLU</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"1189.4\" cy=\"-127.5\" rx=\"36.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"1189.4\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">ReLU</text>\n",
       "</g>\n",
       "<!-- 280619742517376&#45;&gt;280619742520112ReLU -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>280619742517376&#45;&gt;280619742520112ReLU</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1117.46,-127.5C1126.2,-127.5 1134.77,-127.5 1142.77,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1142.91,-131 1152.91,-127.5 1142.91,-124 1142.91,-131\"/>\n",
       "</g>\n",
       "<!-- 280619742517376+ -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>280619742517376+</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"867\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"867\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">+</text>\n",
       "</g>\n",
       "<!-- 280619742517376+&#45;&gt;280619742517376 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>280619742517376+&#45;&gt;280619742517376</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M894.21,-127.5C901.73,-127.5 910.43,-127.5 919.67,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"919.89,-131 929.89,-127.5 919.89,-124 919.89,-131\"/>\n",
       "</g>\n",
       "<!-- 280619742519968 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>280619742519968</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 178,-91.5 178,-55.5 0,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 5.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-55.5 83,-91.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-69.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 3.000</text>\n",
       "</g>\n",
       "<!-- 280619742519968&#45;&gt;280619742519824* -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>280619742519968&#45;&gt;280619742519824*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.12,-73.5C187.07,-73.5 195.75,-73.5 203.65,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.89,-77 213.89,-73.5 203.89,-70 203.89,-77\"/>\n",
       "</g>\n",
       "<!-- 280619742527168 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>280619742527168</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 178,-146.5 178,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 4.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-110.5 83,-146.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-124.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 2.000</text>\n",
       "</g>\n",
       "<!-- 280619742527168&#45;&gt;280619742525008* -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>280619742527168&#45;&gt;280619742525008*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.12,-128.5C187.07,-128.5 195.75,-128.5 203.65,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.89,-132 213.89,-128.5 203.89,-125 203.89,-132\"/>\n",
       "</g>\n",
       "<!-- 280619742520544 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>280619742520544</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"617,-82.5 617,-118.5 804,-118.5 804,-82.5 617,-82.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"663\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 23.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"709,-82.5 709,-118.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"756.5\" y=\"-96.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742520544&#45;&gt;280619742517376+ -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>280619742520544&#45;&gt;280619742517376+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M804.05,-116.67C813.34,-118.3 822.33,-119.87 830.44,-121.29\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"830.1,-124.78 840.55,-123.05 831.3,-117.88 830.1,-124.78\"/>\n",
       "</g>\n",
       "<!-- 280619742520544+&#45;&gt;280619742520544 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>280619742520544+&#45;&gt;280619742520544</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M581.21,-100.5C588.73,-100.5 597.43,-100.5 606.67,-100.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"606.89,-104 616.89,-100.5 606.89,-97 606.89,-104\"/>\n",
       "</g>\n",
       "<!-- 280619742520592 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>280619742520592</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"621.5,-137.5 621.5,-173.5 799.5,-173.5 799.5,-137.5 621.5,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"663\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 6.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"704.5,-137.5 704.5,-173.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"752\" y=\"-151.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742520592&#45;&gt;280619742517376+ -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>280619742520592&#45;&gt;280619742517376+</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M799.58,-139.54C810.51,-137.56 821.15,-135.63 830.59,-133.92\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"831.48,-137.31 840.69,-132.09 830.23,-130.43 831.48,-137.31\"/>\n",
       "</g>\n",
       "<!-- 280619742520112 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>280619742520112</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1261.79,-109.5 1261.79,-145.5 1448.79,-145.5 1448.79,-109.5 1261.79,-109.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1307.79\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 29.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1353.79,-109.5 1353.79,-145.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"1401.29\" y=\"-123.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 1.000</text>\n",
       "</g>\n",
       "<!-- 280619742520112ReLU&#45;&gt;280619742520112 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>280619742520112ReLU&#45;&gt;280619742520112</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M1225.96,-127.5C1233.76,-127.5 1242.39,-127.5 1251.39,-127.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1251.67,-131 1261.67,-127.5 1251.67,-124 1251.67,-131\"/>\n",
       "</g>\n",
       "<!-- 280619378507744 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>280619378507744</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 178,-36.5 178,-0.5 0,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">val 3.000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"83,-0.5 83,-36.5 \"/>\n",
       "<text text-anchor=\"middle\" x=\"130.5\" y=\"-14.8\" font-family=\"Times,serif\" font-size=\"14.00\">grad 5.000</text>\n",
       "</g>\n",
       "<!-- 280619378507744&#45;&gt;280619742519824* -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>280619378507744&#45;&gt;280619742519824*</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M149.46,-36.5C159.06,-39.67 168.84,-43.06 178,-46.5 188.22,-50.34 199.2,-54.95 209.03,-59.27\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"207.86,-62.58 218.42,-63.45 210.71,-56.19 207.86,-62.58\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0xff38e011de80>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# neural network expression & backpropagation\n",
    "from tinyflow import Scalar\n",
    "\n",
    "x1 = Scalar(2.0)\n",
    "x2 = Scalar(3.0)\n",
    "w1 = Scalar(4.0)\n",
    "w2 = Scalar(5.0)\n",
    "b = Scalar(6.0)\n",
    "\n",
    "h = x1 * w1 + x2 * w2 + b\n",
    "y = h.relu()\n",
    "\n",
    "y.backward()\n",
    "draw_dot(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Neural Network with TinyFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Scalar(val=2309136.7873223396, grad=0)\n",
      "1 Scalar(val=201.3856998285673, grad=0)\n",
      "2 Scalar(val=197.4620570963264, grad=0)\n",
      "3 Scalar(val=193.6164948544571, grad=0)\n",
      "4 Scalar(val=189.84745930120087, grad=0)\n",
      "5 Scalar(val=186.1534275554546, grad=0)\n",
      "6 Scalar(val=182.53290704144865, grad=0)\n",
      "7 Scalar(val=178.9844348856714, grad=0)\n",
      "8 Scalar(val=175.506577325794, grad=0)\n",
      "9 Scalar(val=172.0979291313583, grad=0)\n",
      "10 Scalar(val=168.75711303599184, grad=0)\n",
      "11 Scalar(val=165.48277918092327, grad=0)\n",
      "12 Scalar(val=162.2736045695704, grad=0)\n",
      "13 Scalar(val=159.12829253298347, grad=0)\n",
      "14 Scalar(val=156.04557220592474, grad=0)\n",
      "15 Scalar(val=153.0241980133744, grad=0)\n",
      "16 Scalar(val=150.0629491672559, grad=0)\n",
      "17 Scalar(val=147.160629173175, grad=0)\n",
      "18 Scalar(val=144.31606534697642, grad=0)\n",
      "19 Scalar(val=141.52810834091915, grad=0)\n",
      "20 Scalar(val=138.79563167928237, grad=0)\n",
      "21 Scalar(val=136.11753130321225, grad=0)\n",
      "22 Scalar(val=133.49272512462588, grad=0)\n",
      "23 Scalar(val=130.9201525889934, grad=0)\n",
      "24 Scalar(val=128.39877424682, grad=0)\n",
      "25 Scalar(val=125.92757133365585, grad=0)\n",
      "26 Scalar(val=123.50554535846369, grad=0)\n",
      "27 Scalar(val=121.13171770017782, grad=0)\n",
      "28 Scalar(val=118.80512921229185, grad=0)\n",
      "29 Scalar(val=116.52483983531485, grad=0)\n",
      "30 Scalar(val=114.2899282169396, grad=0)\n",
      "31 Scalar(val=112.09949133977008, grad=0)\n",
      "32 Scalar(val=109.95264415645626, grad=0)\n",
      "33 Scalar(val=107.84851923209035, grad=0)\n",
      "34 Scalar(val=105.78626639371933, grad=0)\n",
      "35 Scalar(val=103.76505238683185, grad=0)\n",
      "36 Scalar(val=101.78406053868146, grad=0)\n",
      "37 Scalar(val=99.8424904283093, grad=0)\n",
      "38 Scalar(val=97.9395575631335, grad=0)\n",
      "39 Scalar(val=96.07449306197472, grad=0)\n",
      "40 Scalar(val=94.24654334438901, grad=0)\n",
      "41 Scalar(val=92.4549698261832, grad=0)\n",
      "42 Scalar(val=90.69904862098973, grad=0)\n",
      "43 Scalar(val=88.97807024777957, grad=0)\n",
      "44 Scalar(val=87.29133934419637, grad=0)\n",
      "45 Scalar(val=85.63817438559444, grad=0)\n",
      "46 Scalar(val=84.01790740966867, grad=0)\n",
      "47 Scalar(val=82.42988374656379, grad=0)\n",
      "48 Scalar(val=80.87346175435478, grad=0)\n",
      "49 Scalar(val=79.34801255979073, grad=0)\n",
      "50 Scalar(val=77.85291980419842, grad=0)\n",
      "51 Scalar(val=76.38757939444244, grad=0)\n",
      "52 Scalar(val=74.9513992588406, grad=0)\n",
      "53 Scalar(val=73.54379910793728, grad=0)\n",
      "54 Scalar(val=72.16421020003686, grad=0)\n",
      "55 Scalar(val=70.81207511140371, grad=0)\n",
      "56 Scalar(val=69.48684751103436, grad=0)\n",
      "57 Scalar(val=68.18799193991231, grad=0)\n",
      "58 Scalar(val=66.91498359465564, grad=0)\n",
      "59 Scalar(val=65.66730811546955, grad=0)\n",
      "60 Scalar(val=64.44446137831929, grad=0)\n",
      "61 Scalar(val=63.24594929123829, grad=0)\n",
      "62 Scalar(val=62.071287594690226, grad=0)\n",
      "63 Scalar(val=60.92000166590348, grad=0)\n",
      "64 Scalar(val=59.79162632709954, grad=0)\n",
      "65 Scalar(val=58.685705657537845, grad=0)\n",
      "66 Scalar(val=57.60179280930039, grad=0)\n",
      "67 Scalar(val=56.5394498267429, grad=0)\n",
      "68 Scalar(val=55.49824746953829, grad=0)\n",
      "69 Scalar(val=54.477765039242044, grad=0)\n",
      "70 Scalar(val=53.4775902093087, grad=0)\n",
      "71 Scalar(val=52.497318858491035, grad=0)\n",
      "72 Scalar(val=51.536554907554624, grad=0)\n",
      "73 Scalar(val=50.594910159241834, grad=0)\n",
      "74 Scalar(val=49.67200414142051, grad=0)\n",
      "75 Scalar(val=48.7674639533538, grad=0)\n",
      "76 Scalar(val=47.88092411502966, grad=0)\n",
      "77 Scalar(val=47.0120264194881, grad=0)\n",
      "78 Scalar(val=46.16041978808787, grad=0)\n",
      "79 Scalar(val=45.32576012865249, grad=0)\n",
      "80 Scalar(val=44.50771019643988, grad=0)\n",
      "81 Scalar(val=43.705939457878294, grad=0)\n",
      "82 Scalar(val=42.920123957014084, grad=0)\n",
      "83 Scalar(val=42.14994618461709, grad=0)\n",
      "84 Scalar(val=41.39509494989076, grad=0)\n",
      "85 Scalar(val=40.6552652547355, grad=0)\n",
      "86 Scalar(val=39.93015817051385, grad=0)\n",
      "87 Scalar(val=39.21948071726818, grad=0)\n",
      "88 Scalar(val=38.522945745342106, grad=0)\n",
      "89 Scalar(val=37.840271819357376, grad=0)\n",
      "90 Scalar(val=37.17118310449975, grad=0)\n",
      "91 Scalar(val=36.515409255067766, grad=0)\n",
      "92 Scalar(val=35.87268530523949, grad=0)\n",
      "93 Scalar(val=35.242751562012785, grad=0)\n",
      "94 Scalar(val=34.6253535002763, grad=0)\n",
      "95 Scalar(val=34.02024165996838, grad=0)\n",
      "96 Scalar(val=33.42717154528257, grad=0)\n",
      "97 Scalar(val=32.845903525879024, grad=0)\n",
      "98 Scalar(val=32.27620274006159, grad=0)\n",
      "99 Scalar(val=31.717838999881934, grad=0)\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from tinyflow import Scalar\n",
    "from tinyflow import MLP, MSELoss\n",
    "\n",
    "N, D_in, D_h, D_out = 50, 100, 16, 10\n",
    "\n",
    "def random_matrix(n, d):\n",
    "    x = []\n",
    "    for i in range(n):\n",
    "        x.append([])\n",
    "        for j in range(d):\n",
    "            x[i].append(Scalar(random.random()))  # This is correct\n",
    "    return x\n",
    "\n",
    "# random n * d_in Scalar matrix\n",
    "x = random_matrix(N, D_in)\n",
    "y = random_matrix(N, D_out)\n",
    "\n",
    "model = MLP(D_in, [D_h, D_out])\n",
    "loss_fn = MSELoss()\n",
    "\n",
    "learning_rate = 1e-3\n",
    "for t in range(100):\n",
    "    loss = Scalar(0)\n",
    "    for i in range(N):\n",
    "        y_pred = model(x[i])\n",
    "        loss = loss + loss_fn(y_pred, y[i])\n",
    "\n",
    "    print(t, loss)\n",
    "    model.zero_grad()\n",
    "    loss.backward()\n",
    "    # with no_grad & update weights\n",
    "    for p in model.param():\n",
    "        p.val -= learning_rate * p.grad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "archer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
